#!/usr/bin/python
# coding=utf-8

class simhash:

    #构造函数
    def __init__(self, tokens='', hashbits=128):
        self.hashbits = hashbits
        self.hash = self.simhash(tokens);

    #toString函数
    def __str__(self):
        return str(self.hash)

    #生成simhash值
    def simhash(self, tokens):
        v = [0] * self.hashbits
        for t in [self._string_hash(x) for x in tokens]: #t为token的普通hash值
            for i in range(self.hashbits):
                bitmask = 1 << i
                if t & bitmask :
                    v[i] += 1 #查看当前bit位是否为1,是的话将该位+1
                else:
                    v[i] -= 1 #否则的话,该位-1
        fingerprint = 0
        for i in range(self.hashbits):
            if v[i] >= 0:
                fingerprint += 1 << i
        return fingerprint #整个文档的fingerprint为最终各个位>=0的和

    #求海明距离
    def hamming_distance(self, other):
        x = (self.hash ^ other.hash) & ((1 << self.hashbits) - 1)
        tot = 0;
        while x :
            tot += 1
            x &= x - 1
        return tot

    #求相似度
    def similarity (self, other):
        a = float(self.hash)
        b = float(other.hash)
        if a > b : return b / a
        else: return a / b

    #针对source生成hash值   (一个可变长度版本的Python的内置散列)
    def _string_hash(self, source):
        if source == "":
            return 0
        else:
            x = ord(source[0]) << 7
            m = 1000003
            mask = 2 ** self.hashbits - 1
            for c in source:
                x = ((x * m) ^ ord(c)) & mask
            x ^= len(source)
            if x == -1:
                x = -2
            return x

def sent_similarity (sent1_simhash, sent2_simhash):
    a = float(sent1_simhash)
    b = float(sent2_simhash)
    if a > b : return b / a
    else: return a / b

def compute_simhash(origin_path, result_path):
    f = open(origin_path, 'r', encoding='utf-8')
    data_simhash_file = open(result_path, 'w')
    while True:
        content = f.readline()
        content = content.rstrip('\n')
        if content == '':
            break
        content_hash = simhash(content)
        data_simhash_file.write(content + ',' + str(content_hash) + '\n')
    f.close()
    data_simhash_file.close()

def compare(path):
    f = open(path, 'r', encoding='utf-8')
    result_list = []
    #while True:
    data_list = f.readlines()
    for i in range(len(data_list)):
        sent1 = data_list[i]
        sent1_list = sent1.split(',')
        sentence1 = sent1_list[0]
        hash1 = sent1_list[1].rstrip('\n')
        for j in range(len(data_list)):
            if j > i:
                print('开始第 %s 轮数据处理，%s' % (i, j))
                sent2 = data_list[j]
                sent2_list = sent2.split(',')
                sentence2 = sent2_list[0]
                hash2 = sent2_list[1].rstrip('\n')
                simhash = sent_similarity(hash1, hash2)
                if simhash >= 0.7:
                    if sentence1 not in result_list:
                        result_list.append(sentence1)
                        #result_file.write(sentence1 + '\n')
                else:
                    if sentence1 not in result_list:
                        result_list.append(sentence1)
                        #result_file.write(sentence1 + '\n')
                    if sentence2 not in result_list:
                        result_list.append(sentence2)
                        #result_file.write(sentence2 + '\n')
    print('Success-数据处理完成！')
    f.close()
    result_file = open('result.txt', 'w')
    print('有效数据共有%s条' % len(result_list))
    for n in result_list:
        result_file.write(n + '\n')
    result_file.close()
    print('数据写入成功')

if __name__ == '__main__':
    #s1 = '注册一个QQ邮箱'
    #hash1 = simhash(s1.split())
    #s2 = '申请一个QQ邮箱'
    #hash2 = simhash(s2.split())
    #print(hash1.similarity(hash2))
    #print(sent_similarity(str(hash1), str(hash2)))

    #print('hash1=%s ,hash2=%s' % (hash1, hash2))
    #print(hash1.hamming_distance(hash2) , "   " , hash1.similarity(hash2))
    #origin_path = 'data.txt'
    #result_path = 'data_simhash.txt'
    #print('开始计算数据的simhash值')
    #compute_simhash(origin_path, result_path)
    #print('计算数据的simhash值完成')

    #第二步
    path = 'data_simhash1.txt'
    compare(path)